package com.a.b.c.d.e.webmagic;

import com.a.b.c.d.e.webmagic.processors.BaiduPageProcessor;
import com.a.b.c.d.e.webmagic.processors.GupiaoProcessor;
import com.a.b.c.d.e.webmagic.processors.NewsProcessor;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

/**
 * Created on 2021/9/15.
 *
 * @author yanshuchun
 */
public class SpiderDo {

    public static void main(String[] args) {

        news();
    }
    public static void  news(){
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        Spider spider = Spider.create(new NewsProcessor())
                .setDownloader(httpClientDownloader)
                .addUrl("http://zjj.liaocheng.gov.cn/")
//                .thread(5)
                .addPipeline(new ConsolePipeline());
        spider.run();
        spider.stop();
    }


    public static void gupiao(){
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        Spider spider = Spider.create(new GupiaoProcessor())
                .setDownloader(httpClientDownloader)
                .addUrl("https://www.banban.cn/gupiao/list_sz.html")
                .thread(5)
                .addPipeline(new ConsolePipeline());
        spider.run();
        spider.stop();
    }
    public static void voide(){
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(
                new Proxy("186.96.110.154", 999)

        ));
        Spider spider = Spider.create(new BaiduPageProcessor())
                .setDownloader(httpClientDownloader)
                .addUrl("https://www.ygdy8.com/html/gndy/dyzz/list_23_1.html")
//        .addUrl("https://www.ygdy8.com/html/gndy/dyzz/20210826/61783.html")
//        .addUrl("https://www.ygdy8.com/html/gndy/dyzz/20110920/34259.html")
                .thread(5)
                .addPipeline(new ConsolePipeline());
        spider.run();
        spider.stop();
    }

}
